/*
 * EvolutionProcess.java
 *
 * Created on den 23 oktober 2007, 11:22
 *
 * To change this template, choose Tools | Template Manager
 * and open the template in the editor.
 */
package grex;

import grex.WekaModels.GrexJ48;
import grex.WekaModels.GrexJ48Bag;
import grex.WekaModels.GrexLinearRegression;
import grex.WekaModels.GrexM5P;
import grex.WekaModels.GrexM5PBag;
import grex.WekaModels.GrexM5PRegTree;
import grex.WekaModels.GrexMLP;
import grex.WekaModels.GrexPaceRegression;
import grex.WekaModels.GrexRBF;
import grex.WekaModels.GrexREPTree;
import grex.WekaModels.GrexRandomForest;
import grex.WekaModels.GrexSVMReg;
import grex.WekaModels.GrexSVMClass;
import grex.WekaModels.GrexSimpleCart;
import grex.WekaModels.GrexZeroR;
import grex.WekaModels.WekaPredictiveModel;
import grex.IO.ResultSaver;
import grex.IO.ResultSaverTest;
import grex.ErrorManagement.ErrorManager;
import grex.ErrorManagement.EvolutionListener;
import grex.Data.Prediction;
import grex.Data.PredictionContainer;
import grex.Data.ArffTableModel;

import grex.Data.ExtractionArffTableModel;

import grex.Nodes.GeneException;
import grex.fitnessfunctions.ErrorFunctions.ErrorCalculator;

import java.io.IOException;
import java.io.Serializable;
import java.text.NumberFormat;
import java.util.*;
import java.util.concurrent.CopyOnWriteArrayList;

import wekaGrexBridge.J48Injection;

import wekaGrexBridge.WekaArffTableModel;

/**
 *
 * @author RIK
 */
public class EvolutionProcess implements Runnable, Serializable {

    ArrayList<EvolutionListener> evolutionListeners = new ArrayList<EvolutionListener>();
    CopyOnWriteArrayList<SimplificationListener> simplificationListeners = new CopyOnWriteArrayList<SimplificationListener>();
    ArrayList<GP> batchGPs, selectedGPs;
    HashMap<String, ArrayList<IPredictiveModel>> predictiveModels = new HashMap<String, ArrayList<IPredictiveModel>>();
    private IPredictiveModel simplificationModel;
    
    public void setSimplificationModel(IPredictiveModel model){
        simplificationModel = model;
    }
    
    public ArrayList<GP> getBatchGPs() {
        return batchGPs;
    }

    public void setBatchGPs(ArrayList<GP> batchGPs) {
        this.batchGPs = batchGPs;
    }
    private Options options;
    Simplyfier simplyfier = null;
    GP bestGP, bestSimplyfiedGP, publicGP, bestSelectedGP;
    private int currentGeneration;
    int currentBatch, persistence = 0;
    Hashtable<String, EvolutionResult> foldResults, selectedFoldResults;
    World world;
    Thread t;
    private ArffTableModel data;
    private String summary = "", selectedSummary = "";
    NumberFormat nf = NumberFormat.getInstance(Locale.ENGLISH);
    GP batchWinner;
    ErrorCalculator ec;
    private Environment environment;
    private final int TREE_INJECTION_INTERVAL = 5,
            NR_OF_INJECTED_MUTADED_TREES = 5;
    HashMap<String, ArrayList<GP>> foldGPs = new HashMap<String, ArrayList<GP>>();
    /**
     * Creates a new instance of EvolutionProcess
     */
    
    private IPredictiveModel extractionModel;
    public EvolutionProcess(ArffTableModel data, Options options) {
        try {
            options.setNR_OF_INPUT_COLUMNS(data.getNrOfCategorialColumns() + data.getNrOfContinousColumns());
            this.data = data;
            world = new World();
            this.options = options;
            nf.setMinimumFractionDigits(4);
            //   this.addEvolutionListener(new ConsoleOutputHandler());
        } catch (BNFException ex) {
            ErrorManager.getInstance().reportError(ex);
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        } catch (IOException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
    }
    public EvolutionProcess(ArffTableModel data, Options options,IPredictiveModel extractionModel) {
        this(data,options);
        this.extractionModel = extractionModel;
    }



    public void startEvolution(Options options) {
        this.options = options;
        t = new Thread(this);
        t.start();
    }

    public void startEvolution() {
        t = new Thread(this);
        t.start();
    }

    public void startEvolution(String dataFile, String BNFfile) {
        try {
            t = new Thread(this);

            environment = new Environment(dataFile, BNFfile);
            this.options = environment.getOptions();
            this.data = environment.getData();
            getOptions().setNR_OF_INPUT_COLUMNS(getData().getNrOfCategorialColumns() + getData().getNrOfContinousColumns());
            world.createNewPopulation(new Environment(getOptions()));
            t.start();
        } catch (BNFException ex) {
            ErrorManager.getInstance().reportError(ex);
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        } catch (IOException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
    }

    String lastTestFold = "";
    ResultSaver resultSaver;
    ResultSaverTest resultSaverTest;
    ArrayList<String> modelNames = new ArrayList<String>();
    private ArffTableModel orgData;
    public static ExtractionArffTableModel extractionData=null;
    public void run() {
        try {
            orgData = data;
            
                // Rule Extraction
            if(extractionModel!=null){
                 extractionData = new ExtractionArffTableModel(data, extractionModel);
                    extractionData.setUseOracle(options.isORACLE_COACHING_TEST());
                    data = extractionData;
                   // orgData = extractionData;
            }
             /*   
            if (options.isRULE_EXTRACT_FROM_RF() && options.getPROBLEM_TYPE() == Options.CLASSIFICATION) {
                    WekaPredictiveModel model = new GrexJ48Bag(orgData);// GrexRandomForest(data);
                    ExtractionArffTableModel extractionData = new ExtractionArffTableModel(data, model);
                    extractionData.setUseOracle(options.isORACLE_COACHING_TEST());
                    data = extractionData;
                    orgData = extractionData;
                }

                if (options.isRULE_EXTRACTION_FROM_M5PBag() && options.getPROBLEM_TYPE() == Options.REGRESSION) {
                    WekaPredictiveModel model = new GrexM5PBag(data);
                    ExtractionArffTableModel extractionData = new ExtractionArffTableModel(data, model);
                    extractionData.setUseOracle(options.isORACLE_COACHING_TEST());
                    data = extractionData;
                }  
             */





            String startFold = getOptions().getTestFOLD();
            summary = "";
            foldResults = new Hashtable<String, EvolutionResult>();
            selectedFoldResults = new Hashtable<String, EvolutionResult>();
            environment = new Environment(data, getOptions());
            ec = new ErrorCalculator(environment);

            addEvolutionListener(resultSaver);



            ArrayList<String> arlFolds;
            if (getOptions().getTestFOLD().equalsIgnoreCase("ALL")) {
                arlFolds = environment.getFolds();
            } else {
                arlFolds = new ArrayList<String>();
                arlFolds.add(getOptions().getTestFOLD());
            }

            long startTime = System.currentTimeMillis();
            double orgLengthPunishment = options.getPUNISHMENT_FOR_LENGTH();
            double originalTombolaPercent = options.getTOMBOLA_PERCENT();
            for (int f = 0; f < arlFolds.size()&& !t.isInterrupted(); f++) {

                if (t.isInterrupted()) {
                    break;
                }
                String fold = arlFolds.get(f);
                batchGPs = new ArrayList<GP>();
                selectedGPs = new ArrayList<GP>();
                getOptions().setTestFOLD(fold);


                getOptions().setWinners(null);
                
                
                if (options.isEXP_VAL_FOLDS()) {
                    int valFoldIndex = f + 1;
                    if (valFoldIndex == arlFolds.size()) {
                        valFoldIndex = 0;
                    }
                    String valFold = arlFolds.get(valFoldIndex);
                    options.setVAL_FOLD(valFold);
                    data.setValFold(valFold);
                }else{
                    data.setValFold(options.getVAL_FOLD());
                    data.initFold(fold);
                }

                
   

                
                
                initBenchmarkTechniques(fold);





                if (options.isRUN_GREX()) {

                    int batchSize = getOptions().getBATCH_SIZE();
                    int nrOfgenerations = getOptions().getGENERATIONS();
                    ArrayList<GP> injectionGPs = null;
                    double injectionGPAvgOutOfBagAcc = 0, injectionGPBootStrapAcc = 0;
                    double injectionGPReasonableACC = 0;
                    int maxTreeSize = options.getMAX_TREE_SIZE();
                    for (currentBatch = 0; currentBatch < batchSize && !t.isInterrupted(); currentBatch++) {
                        if (t.isInterrupted()) {
                            break;
                        }
                        options.setPUNISHMENT_FOR_LENGTH(orgLengthPunishment);
                        if (getOptions().isUSE_GP_INJECTION()) {                                    
                                WekaArffTableModel m = new WekaArffTableModel(new J48Injection(),data);// (WekaArffTableModel) getData();                               
                                injectionGPReasonableACC = m.calcReasonableAcc(10, 0.1f,fold);
                                injectionGPs = new ArrayList<GP>();
                                for (int i = 1; i < options.getGENERATIONS() / TREE_INJECTION_INTERVAL + 3; i++) {
                                    GP j48 = m.createJ48GP(0.05f, environment,fold);
                                    j48.cutTree(i);
                                    j48.train();
                                    System.out.println("Length: " + j48.getNrOfNodes() + "Acc" + j48.calcTrainError("ACC"));// errorCalculator.getACC().calcError(j48, j48.pcTrain));// j48.getTrainAcc());
                                    injectionGPs.add(0, j48); // to be compatible with jaTI that sorts the tre in descending order.
                                    j48.getHead().notifyChildrenOfModification();                                    
                                }
  
                            injectionGPReasonableACC = (1-injectionGPReasonableACC) * 100;
                            getOptions().setTARGET_ACCURACY(injectionGPReasonableACC);
                            world.createNewPopulation(environment);
                        }  else {
                            world.createNewPopulation(environment);
                        }

                        persistence = options.getSTOP_WITHOUT_IMPROVMENT();
                        bestGP = world.getBestEverGP();
                        double bestFitness = bestGP.getFitness();
                        options.setTOMBOLA_PERCENT(originalTombolaPercent);

                        this.fireEvolutionStarted();

                        for (currentGeneration = 0; getCurrentGeneration() <= nrOfgenerations && !t.isInterrupted() && persistence != -1; currentGeneration++) {
                            if (t.isInterrupted()) {
                                break;
                            }
                            if (getCurrentGeneration() == nrOfgenerations) {
                                options.setTOMBOLA_PERCENT(1);
                            }
                            if (4 * getCurrentGeneration() < options.getGENERATIONS()) {
                                options.setMAX_TREE_SIZE(1000);
                            } else {
                                options.setMAX_TREE_SIZE(maxTreeSize);
                            }
                            bestGP = (GP) world.getPopulation().getBestGP().clone();
                            bestGP.calcAdjustedFitness();
                            //bestGP.optimize();
                            bestGP.clean();
                            world.evolve(1);
                            if (bestGP.getFitness() < bestFitness) {
                                persistence = options.getSTOP_WITHOUT_IMPROVMENT();
                                bestFitness = bestGP.getFitness();
                            }
                            this.fireGenerationTerminated();
                            if (options.getSTOP_WITHOUT_IMPROVMENT() != 0) {
                                persistence--;
                            }
                            //  bestGP.getFitnessFunction().


                            if (getOptions().isUSE_GP_INJECTION() && injectionGPs != null && (getCurrentGeneration()) % (TREE_INJECTION_INTERVAL * 2) == 0) {

                                if (bestGP.getTrainAccError() > injectionGPReasonableACC) {
                                    getOptions().setPUNISHMENT_FOR_LENGTH(getOptions().getPUNISHMENT_FOR_LENGTH() * 0.9);
                                    System.out.println("Decreased Length Punishmet:" + getOptions().getPUNISHMENT_FOR_LENGTH());
                                }
                            }

                            if (getOptions().isUSE_GP_INJECTION() && injectionGPs != null && getCurrentGeneration() % TREE_INJECTION_INTERVAL == 0) {
                                System.out.println("BestAcc: " + bestGP.getTrainAccError() + " ReasonableACC: " + injectionGPReasonableACC);
                                if (bestGP.getTrainAccError() > injectionGPReasonableACC) {
                                    if (injectionGPs.size() > 0) {
                                        GP gp = injectionGPs.get(injectionGPs.size() - 1);
                                        System.out.println("Injection!!!!! GP Length: " + gp.getNrOfNodes());
                                        world.getPopulation().injectGP(injectionGPs.get(injectionGPs.size() - 1), NR_OF_INJECTED_MUTADED_TREES);
                                        injectionGPs.remove(injectionGPs.size() - 1);
                                    }
 
                                }
                            }
                            
                        }
                        batchWinner = (GP) world.getBestEverGP().clone();
                        batchWinner.clean();
                        batchGPs.add(batchWinner);

                        fireBatchTerminated();

                    }
                    bestGP = getBestGpFromBatch(batchGPs);
                    addPredicitiveModel(bestGP);

                    if (getOptions().isSIMPLIFICATION()) {
                        startSimplification();
                    }
                    foldGPs.put(fold, batchGPs);
                    lastTestFold = fold;
                    this.fireEvolutionTerminated();

                }//End RUN G-REX

                summary = getMeanResults() + getResults();//getStatSummary(arlFolds);
                getOptions().setTestFOLD(startFold);

            }
            

            fireEvolutionProcessTerminated();
            evolutionListeners.remove(resultSaver);

            System.out.println("Time: " + (System.currentTimeMillis() - startTime));
        } catch (Exception e) {
            e.printStackTrace();
            ErrorManager.getInstance().reportError(e);
            this.fireEvolutionTerminated();
        }

    }
        private void initBenchmarkTechniques(String fold) {
        if (options.getPROBLEM_TYPE() == Options.REGRESSION) {
            if (options.isZeroRReg()) {
                GrexZeroR zero = new GrexZeroR(orgData);
                zero.initFold(fold);
                zero.train();
                addPredicitiveModel(zero);
            }
            //  */
            if (options.isSVMReg()) {
                GrexSVMReg svmReg = new GrexSVMReg(orgData);
                svmReg.initFold(fold);
                svmReg.train();
                addPredicitiveModel(svmReg);//*/
            }

            if (options.isRBF()) {
                GrexRBF rbf = new GrexRBF(orgData);
                rbf.initFold(fold);
                rbf.train();
                addPredicitiveModel(rbf);//*/
            }

            if (options.isM5PModelTree()) {
                GrexM5P m5P = new GrexM5P(orgData);
                m5P.initFold(fold);
                m5P.train();
                addPredicitiveModel(m5P);//*/
            }

            if (options.isM5PRegTree()) {
                GrexM5PRegTree m5P = new GrexM5PRegTree(orgData);
                m5P.initFold(fold);
                m5P.train();
                addPredicitiveModel(m5P);//*/
            }

            if (options.isMLPReg()) {
                GrexMLP mlp = new GrexMLP(orgData);
                mlp.initFold(fold);
                mlp.train();
                addPredicitiveModel(mlp);//*/
            }
            if (options.isReptreeREG()) {
                GrexREPTree tree = new GrexREPTree(orgData);
                tree.initFold(fold);
                tree.train();
                addPredicitiveModel(tree);//*/
            }

            if (options.isLinearRegression()) {
                GrexLinearRegression reg = new GrexLinearRegression(orgData);
                reg.initFold(fold);
                reg.train();
                addPredicitiveModel(reg);
            }

            if (options.isPaceRegression()) {
                GrexPaceRegression pace = new GrexPaceRegression(orgData);

                pace.initFold(fold);
                pace.train();
                addPredicitiveModel(pace);
            }

            if (options.isBaggedM5P()) {
                GrexM5PBag bag = new GrexM5PBag(orgData);
                bag.initFold(fold);
                bag.train();
                addPredicitiveModel(bag);
            }
            /*
             * GrexWekaEnsemble ens = new GrexWekaEnsemble(orgData);
             * ens.initFold(fold); // (()) reg.train(); ens.addMember(m5P);
             * ens.addMember(tree); ens.addMember(reg); // ens.addMember(mlp);
             * addPredicitiveModel(ens); //
             */


        } else {//Classification           

            System.out.println("\n--------------------------------");
            if (options.isZeroR()) {
                GrexZeroR zero = new GrexZeroR(orgData);
                zero.initFold(fold);
                zero.train();
                addPredicitiveModel(zero);
            }

            if (options.isJ48()) {
                GrexJ48 j48 = new GrexJ48(orgData);
                j48.initFold(fold);
                j48.train();
                addPredicitiveModel(j48);
            }

            if (options.isSimpleCart()) {
                GrexSimpleCart scart = new GrexSimpleCart(orgData);
                scart.initFold(fold);
                scart.train();
                addPredicitiveModel(scart);
            }
            
            if (options.isREPTree()) {
                GrexREPTree rTree = new GrexREPTree(orgData);
                rTree.initFold(fold);
                rTree.train();
                addPredicitiveModel(rTree);
            }

            if (options.isRandomForest()) {
                GrexRandomForest rf = new GrexRandomForest(orgData);
                rf.initFold(fold);
                rf.train();
                addPredicitiveModel(rf);
            }
            
            if (options.isBaggingJ48()) {
                GrexJ48Bag bj48 = new GrexJ48Bag(orgData);
                bj48.initFold(fold);
                bj48.train();
                addPredicitiveModel(bj48);
            }

            if (options.isMLP()) {
                GrexMLP mlp = new GrexMLP(orgData);
                mlp.initFold(fold);
                mlp.train();
                addPredicitiveModel(mlp);
            }
            if (options.isSVM()) {
                GrexSVMClass svm = new GrexSVMClass(orgData);
                svm.initFold(fold);
                svm.train();
                addPredicitiveModel(svm);
            }
        }
    }
    
    
    
    public boolean isAlive() {
        return t.isInterrupted();
    }

    private void addPredicitiveModel(IPredictiveModel model) {
        ArrayList<IPredictiveModel> models = predictiveModels.get(model.getName());
        if (models == null) {
            models = new ArrayList<IPredictiveModel>();
            predictiveModels.put(model.getName(), models);
        }
        models.add(model);
        if (!modelNames.contains(model.getName())) {
            modelNames.add(model.getName());
        }
    }    
    


    public GP getBatchWinner() {
        return batchWinner;
    }

    public HashMap<String, ArrayList<IPredictiveModel>> getPredicitveModels() {
        return predictiveModels;
    }

    public String getResults() {
        String results = "\n\n<------------ All results -------------->\n";
        ArrayList<String> keys;


        for (String key : modelNames) {
            try {
                ArrayList<IPredictiveModel> models = predictiveModels.get(key);
                results += key + "\n" + ec.getStatistics(models) + "\n\n";
            } catch (GeneException ex) {
                ErrorManager.getInstance().reportError(ex);
            }
        }
        return results;
    }

    public String getMeanResults() {
        String meanResults = options.getDATA_FILE() + " " + environment.getFolds().size() + "folds:";
        meanResults += "\t" + ec.getHeaders();
        for (String key : modelNames) {
            try {
                ArrayList<IPredictiveModel> models = predictiveModels.get(key);
                meanResults += key + "\t " + ec.getMeanStatistics(models) + "\n";
            } catch (GeneException ex) {
                ErrorManager.getInstance().reportError(ex);
            }
        }
        return meanResults + "\n";
    }
    
        public String getMeanResultsForFile() {
        String meanResults =""; 
       // meanResults += "\t" + ec.getHeaders();
        for (String key : modelNames) {
            try {
                ArrayList<IPredictiveModel> models = predictiveModels.get(key);
                meanResults +=options.getDATA_FILE() +"\t"+ key + "\t " + ec.getMeanStatistics(models) + "\n";
            } catch (GeneException ex) {
                ErrorManager.getInstance().reportError(ex);
            }
        }
        return meanResults;
    }

    public String getBatchResult() throws GeneException {

        String s = "";
        ArrayList<IPredictiveModel> models;
        ArrayList<IPredictiveModel> ipm = new ArrayList<IPredictiveModel>();
        for (String key : modelNames) {
            try {
                models = predictiveModels.get(key);
                s += ec.getStatisitcsAsRow(models);
            } catch (GeneException ex) {
                ErrorManager.getInstance().reportError(ex);
            }
        }
        StringTokenizer st = new StringTokenizer(options.getDATA_FILE(), "\\");
        String name = "";
        while (st.hasMoreTokens()) {
            name = st.nextToken();
        }
        return name + "\t" + s + "\n";
    }

    public String getBatchSensitivity() throws GeneException {

        String s = "";
        ArrayList<IPredictiveModel> models;
        ArrayList<IPredictiveModel> ipm = new ArrayList<IPredictiveModel>();
        for (String key : modelNames) {
            models = predictiveModels.get(key);
            double[] sensitivity = models.get(0).calcSensitivity();
            for (int i = 0; i < sensitivity.length; i++) {
                s += sensitivity[i] + " \t";
            }
            s += "\t";
        }
        StringTokenizer st = new StringTokenizer(options.getDATA_FILE(), "\\");
        String name = "";
        while (st.hasMoreTokens()) {
            name = st.nextToken();
        }
        return name + "\t" + s + "\n";
    }

    public String getBatchSensitivityHeaders() throws GeneException {
        String s = "";
        ArrayList<IPredictiveModel> models;
        for (String key : modelNames) {
            models = predictiveModels.get(key);
            String[] cNames = data.getColumnNames();
            for (int i = 0; i < cNames.length - 1; i++) {
                s += key + "-" + cNames[i] + " \t";
            }
            s += "\t";
        }
        return "Dataset" + "\t" + s + "\n";
    }

    public String getBatchResultPredictions() throws GeneException {


        StringTokenizer st = new StringTokenizer(options.getDATA_FILE(), "\\");
        String name = "";
        while (st.hasMoreTokens()) {
            name = st.nextToken();
        }

        String s = "Prediction for: " + name + "\n";
        ArrayList<IPredictiveModel> models;
        for (String key : modelNames) {
            models = predictiveModels.get(key);
            s += key + "\n";
            s += models.get(0).getOrderedTestPredictions();
        }

        return s + "\n";
    }

    public String getBatchHeaders() {
        return "\t" + ec.getStatisitcsAsRowHeaders();
    }

    public String getHeaders() {
        return "\t" + ec.getHeaders();
    }

    public String getAllPredictions() {
        String preds = "";
        String models = "";
        for (EvolutionResult er : foldResults.values()) {
            ArrayList<ArrayList<Prediction>> trainPreds = new ArrayList<ArrayList<Prediction>>();
            ArrayList<ArrayList<Prediction>> testPreds = new ArrayList<ArrayList<Prediction>>();
            String fold = er.getBatchGPs().get(0).getOptions().getTestFOLD();
            models += "\n-------Models for " + fold + " ----------\n";
            for (GP gp : er.getBatchGPs()) {
                models += gp.toString() + "\n";
                trainPreds.add(gp.getOrderedTrain());
                testPreds.add(gp.getOrderedTest());
            }
            trainPreds.add(er.getEnsTrains());
            testPreds.add(er.getEnsTests());
            preds += "-------Train Predictions " + fold + " ---------\n";
            for (int row = 0; row < trainPreds.get(0).size(); row++) {
                for (int col = 0; col < trainPreds.size(); col++) {
                    preds += trainPreds.get(col).get(row).getPrediction() + "\t";
                }
                preds += trainPreds.get(0).get(row).getTargetValue() + "\n";
            }
            preds += "-------Trest Predictions " + fold + " ---------\n";
            for (int row = 0; row < testPreds.get(0).size(); row++) {
                for (int col = 0; col < testPreds.size(); col++) {
                    preds += testPreds.get(col).get(row).getPrediction() + "\t";
                }
                preds += testPreds.get(0).get(row).getTargetValue() + "\n";
            }

        }

        return preds + models;
    }

    public String getAllTestPredictions() {
        NumberFormat nf = NumberFormat.getInstance();
        nf.setMinimumFractionDigits(4);
        String preds = "";
        String models = "";
        for (EvolutionResult er : foldResults.values()) {
            ArrayList<ArrayList<Prediction>> testPreds = new ArrayList<ArrayList<Prediction>>();
            String fold = er.getBatchGPs().get(0).getOptions().getTestFOLD();
            models += "\n-------Models for " + fold + " ----------\n";
            for (GP gp : er.getBatchGPs()) {
                models += gp.toString() + "\n";
                testPreds.add(gp.getOrderedTest());
            }
            testPreds.add(er.getEnsTests());
            preds += "-------Trest Predictions " + fold + " ---------\n";
            for (int row = 0; row < testPreds.get(0).size(); row++) {
                for (int col = 0; col < testPreds.size(); col++) {
                    preds += testPreds.get(col).get(row).getPrediction() + "\t";
                }
                preds += testPreds.get(0).get(row).getTargetValue() + "\n";
            }

        }

        return preds + models;
    }


    public String getBatchTrainProbs() {

        String probsAndReals = "";
        int n = foldResults.values().iterator().next().getBestGP().getPcTrain().values().iterator().next().getProbs().length;

        String[] prbs = new String[n * 2 + 2 + 2];
        for (int i = 0; i < prbs.length; i++) {
            prbs[i] = "";
        }
        int m = 1;
        int f = 1;
        probsAndReals = "TrainProbs\n";
        for (EvolutionResult er : foldResults.values()) {
            for (GP gp : er.getBatchGPs()) {
                probsAndReals += "F" + f + "\tM" + m + "\tI" + gp.getOrderedTrain().size() + "\n";//.getPcTrain().values().size()+"\n";
                try {

                    for (int i = 0; i < prbs.length; i++) {
                        prbs[i] = "";
                    }
                    for (Prediction p : gp.getOrderedTrain()) {// .getPcTrain().values()) {
                        for (int i = 0; i < n; i++) {
                            prbs[i] += nf.format(p.getProbs()[i]) + "\t";
                        }
                        prbs[n] += p.getPrediction() + "\t";
                        prbs[prbs.length - 1] += p.getTargetValue() + "\t";
                    }
                    for (int i = 0; i < n; i++) {
                        probsAndReals += "P(" + i + ")\t" + prbs[i] + "\n";

                    }
                    probsAndReals += "Pred:\t" + prbs[n] + "\n";

                    probsAndReals += "ACC: " + nf.format(ec.calcError(gp, "TRAIN", "ACC")) + "\n";
                    probsAndReals += "AUC: " + nf.format(ec.calcError(gp, "TRAIN", "AUC")) + "\n";
                    probsAndReals += "VAL: " + nf.format(ec.calcError(gp, "VAL", "ACC")) + "\n";
//------------------------EnsFidelity-----------------------------------------------
                    ArrayList<Prediction> ens = er.getEnsTrains();
                    Prediction eP, gpP;
                    for (int i = 0; i < ens.size(); i++) {
                        eP = ens.get(i);
                        gpP = gp.getPcTrain().get(eP.getInstance());
                        gpP.setTargetValue(eP.getPrediction());
                    }
                    probsAndReals += "EnsFidACC: " + nf.format(ec.calcError(gp, "TRAIN", "ACC")) + "\n";
                    probsAndReals += "EnsFidAUC: " + nf.format(ec.calcError(gp, "TRAIN", "AUC")) + "\n\n";//
//-----------------------------------------------------------------------------------
                } catch (Exception e) {
                    e.printStackTrace();
                }
                m++;
            }
            probsAndReals += "F" + f + " Ens" + "\tI" + er.getBatchGPs().get(0).getOrderedTrain().size() + "\n";//.getPcTrain().values().size()+"\n";
            m = 1;
            ArrayList<Prediction> ensPreds = er.getEnsTrains();
            for (Prediction p : ensPreds) {
                for (int i = 0; i < n; i++) {
                    prbs[n + 2 + i] += nf.format(p.getProbs()[i]) + "\t";
                }
                prbs[prbs.length - 2] += p.getPrediction() + "\t";
            }

            for (int i = n + 2; i < n + 2 + n; i++) {
                probsAndReals += "EnsP(" + (i - n - 2) + ")\t" + prbs[i] + "\n";
            }
            probsAndReals += "EnsPred:\t" + prbs[prbs.length - 2] + "\n";
            probsAndReals += "EnsACC: " + nf.format(er.getEnsTrainACC()) + "\n";
            probsAndReals += "EnsAUC: " + nf.format(er.getEnsTrainAUC()) + "\n";
            probsAndReals += "\nTarget:\t" + prbs[prbs.length - 1] + "\n\n";
            f++;
        }


        return probsAndReals;
    }

    public String getBatchTestProbs() {

        String probsAndReals = "";
        int n = foldResults.values().iterator().next().getBestGP().getPcTest().values().iterator().next().getProbs().length;

        String[] prbs = new String[n * 2 + 2 + 2];
        for (int i = 0; i < prbs.length; i++) {
            prbs[i] = "";
        }
        int m = 1;
        int f = 1;
        probsAndReals = "TestProbs\n";
        for (EvolutionResult er : foldResults.values()) {
            for (GP gp : er.getBatchGPs()) {
                probsAndReals += "F" + f + "\tM" + m + "\tI" + gp.getOrderedTest().size() + "\n";//.getPcTrain().values().size()+"\n";
                try {

                    for (int i = 0; i < prbs.length; i++) {
                        prbs[i] = "";
                    }
                    for (Prediction p : gp.getOrderedTest()) {// .getPcTrain().values()) {
                        for (int i = 0; i < n; i++) {
                            prbs[i] += nf.format(p.getProbs()[i]) + "\t";
                        }
                        prbs[n] += p.getPrediction() + "\t";
                        prbs[prbs.length - 1] += p.getTargetValue() + "\t";
                    }
                    for (int i = 0; i < n; i++) {
                        probsAndReals += "P(" + i + ")\t" + prbs[i] + "\n";

                    }
                    probsAndReals += "Pred:\t" + prbs[n] + "\n";

                    probsAndReals += "ACC: " + nf.format(ec.calcError(gp, "TEST", "ACC")) + "\n";
                    probsAndReals += "AUC: " + nf.format(ec.calcError(gp, "TEST", "AUC")) + "\n";
//------------------------EnsFidelity-----------------------------------------------
                    ArrayList<Prediction> ens = er.getEnsTests();
                    Prediction eP, gpP;
                    for (int i = 0; i < ens.size(); i++) {
                        eP = ens.get(i);
                        gpP = gp.getPcTest().get(eP.getInstance());
                        gpP.setTargetValue(eP.getPrediction());
                    }
                    probsAndReals += "EnsFidACC: " + nf.format(ec.calcError(gp, "TEST", "ACC")) + "\n";
                    probsAndReals += "EnsFidAUC: " + nf.format(ec.calcError(gp, "TEST", "AUC")) + "\n\n";//
//-----------------------------------------------------------------------------------
                } catch (Exception e) {
                    e.printStackTrace();
                }
                m++;
            }
            m = 1;

            probsAndReals += "F" + f + " Ens" + "\tI" + er.getBatchGPs().get(0).getOrderedTest().size() + "\n";//.getPcTrain().values().size()+"\n";
            ArrayList<Prediction> ensPreds = er.getEnsTests();
            for (Prediction p : ensPreds) {
                for (int i = 0; i < n; i++) {
                    prbs[n + 2 + i] += nf.format(p.getProbs()[i]) + "\t";
                }
                prbs[prbs.length - 2] += p.getPrediction() + "\t";
            }

            for (int i = n + 2; i < n + 2 + n; i++) {
                probsAndReals += "EnsP(" + (i - n - 2) + ")\t" + prbs[i] + "\n";
            }
            probsAndReals += "EnsPred:\t" + prbs[prbs.length - 2] + "\n";
            probsAndReals += "EnsACC: " + nf.format(er.getEnsTestACC()) + "\n";
            probsAndReals += "EnsAUC: " + nf.format(er.getEnsTestAUC()) + "\n";

            probsAndReals += "\nTarget:\t" + prbs[prbs.length - 1] + "\n\n";
            f++;
        }


        return probsAndReals;
    }

    public String getBatchALLProbs() {

        String probsAndReals = "";
        int n = foldResults.values().iterator().next().getBestGP().getPcTest().values().iterator().next().getProbs().length;

        String[] prbs = new String[n * 2 + 2 + 2];
        for (int i = 0; i < prbs.length; i++) {
            prbs[i] = "";
        }
        int m = 1;
        int f = 1;
        probsAndReals = "\n\nAllProbs\n";
        for (EvolutionResult er : foldResults.values()) {
            for (GP gp : er.getBatchGPs()) {

                try {
                    ArrayList<Prediction> all = new ArrayList<Prediction>();
                    PredictionContainer pc = gp.getPcTrain();
                    pc.putAll(gp.getPcTest());

                    all.addAll(er.getEnsTrains());
                    all.addAll(er.getEnsTests());

                    for (int i = 0; i < prbs.length; i++) {
                        prbs[i] = "";
                    }

                    //probsAndReals +="ACC: " + nf.format(sACC.calcACC(gp, gp.getPcTest()))+"\n";
                    //probsAndReals +="AUC: " + nf.format(sAUC.calcAUC(gp, gp.getPcTest()))+"\n";
//------------------------EnsFidelity-----------------------------------------------

                    Prediction eP, gpP;
                    for (int i = 0; i < all.size(); i++) {
                        eP = all.get(i);
                        gpP = pc.get(eP.getInstance());
                        gpP.setTargetValue(eP.getPrediction());
                    }
                    probsAndReals += "F" + f + " M" + m + " I" + all.size() + " AllEnsFidACC: " + nf.format(ec.calcError(pc, "ACC")) + "\n";
                    probsAndReals += "F" + f + " M" + m + " I" + all.size() + " AllEnsFidAUC: " + nf.format(ec.calcError(pc, "AUC")) + "\n";//



//-----------------------------------------------------------------------------------
                } catch (Exception e) {
                    e.printStackTrace();
                }
                m++;
            }
            m = 1;
            f++;
            probsAndReals += "\n";
        }


        return probsAndReals;
    }

    public double getBatchTestAUC(Hashtable<String, EvolutionResult> foldResults) {
        try {
            ArrayList<Prediction> all = new ArrayList<Prediction>();
            GP gp = null;
            for (EvolutionResult er : foldResults.values()) {
                gp = er.getBestGpFromBatch();
                //  gp.train();
                gp.execute(gp.getPcTest());
                for (Prediction p : gp.getPcTest().values()) {
                    all.add(p);
                }
            }

            return ec.calcError(all, "AUC");
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
        return -1;
    }

    public String getEnsBatchTrainProbs() {

        String probsAndReals = "";
        int n = foldResults.values().iterator().next().getBestGP().getPcTest().values().iterator().next().getProbs().length;

        String[] prbs = new String[n + 2];
        for (int i = 0; i < prbs.length; i++) {
            prbs[i] = "";
        }
        int m = 1;
        int f = 1;
        probsAndReals = "TrainProbs\n";
        for (EvolutionResult er : foldResults.values()) {
            for (GP gp : er.getBatchGPs()) {
                probsAndReals += "F" + f + "\tM" + m + "\tI" + gp.getPcTrain().values().size() + "\n";
                try {
                    probsAndReals += "ACC: " + nf.format(gp.calcTrainError("ACC")) + "\n";
                    probsAndReals += "AUC: " + nf.format(gp.calcTrainError("AUC")) + "\n";
                } catch (Exception e) {
                    e.printStackTrace();
                }
                for (int i = 0; i < prbs.length; i++) {
                    prbs[i] = "";
                }
                for (Prediction p : gp.getPcTrain().values()) {
                    for (int i = 0; i < p.getProbs().length; i++) {
                        prbs[i] += nf.format(p.getProbs()[i]) + "\t";
                    }
                    prbs[prbs.length - 2] += p.getPrediction() + "\t";
                    prbs[prbs.length - 1] += p.getTargetValue() + "\t";
                }
                for (int i = 0; i < prbs.length - 2; i++) {
                    probsAndReals += "P(" + i + ")\t" + prbs[i] + "\n";

                }
                probsAndReals += "Prediction:\t" + prbs[prbs.length - 2] + "\n";
                probsAndReals += "Target:\t" + prbs[prbs.length - 1] + "\n";
                m++;
            }
            m = 1;
            f++;
        }
        return probsAndReals;
    }

    public double getBatchEnsTestAUC(Hashtable<String, EvolutionResult> foldResults) {
        try {
            ArrayList<Prediction> all = new ArrayList<Prediction>();
            Environment env = foldResults.values().iterator().next().getBestGpFromBatch().getEnvironment();
            for (EvolutionResult er : foldResults.values()) {
                all.addAll(er.getEnsTests());
            }
            return ec.calcError(all, "AUC");
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
        return -1;
    }

    public double getBatchEnsTrainAUC(Hashtable<String, EvolutionResult> foldResults) {
        try {
            List<Prediction> all = new ArrayList<Prediction>();
            Environment env = foldResults.values().iterator().next().getBestGpFromBatch().getEnvironment();
            for (EvolutionResult er : foldResults.values()) {
                all.addAll(er.getEnsTrains());
            }
            return ec.calcError(all, "AUC");
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
        return -1;
    }

    public double getBatchTrainAUC(Hashtable<String, EvolutionResult> foldResults) {
        try {
            ArrayList<Prediction> all = new ArrayList<Prediction>();
            GP gp = null;
            for (EvolutionResult er : foldResults.values()) {
                gp = er.getBestGpFromBatch();
                for (Prediction p : gp.getPcTrain().values()) {
                    all.add(p);
                }
            }

            return ec.calcError(all, "AUC");
        } catch (GeneException ex) {
            ErrorManager.getInstance().reportError(ex);
        }
        return -1;
    }

    public EvolutionResult getBatchResult(String fold) {
        return foldResults.get(fold);
    }

    public EvolutionResult getLastBatchResult() {
        return foldResults.get(lastTestFold);
    }

    public EvolutionResult getSelectedResult(String fold) {
        return selectedFoldResults.get(fold);
    }

    public void stopEvolution() {
        if (simplyfier != null && !simplyfier.isInterrupted()) {
            simplyfier.interrupt();
        }
        if (t != null) {
            t.interrupt();
        }
    }

    public void startSimplification() {
        try {
            simplyfier = new Simplyfier(bestGP, world, simplificationListeners);
            simplyfier.start();
            simplyfier.join();
            // simplyfier.simplify();
            bestSimplyfiedGP = simplyfier.getSimplyfiedGP();
        } catch (Exception e) {
            ErrorManager.getInstance().reportError(e);
        }
    }

    public void createAlternativeSolution(GP gp) {
        try {
            world.getEnvironment().getOptions().setSIMPLIFICATION_FUNCTION("AlternativeSolutionFitness");
            world.getEnvironment().getOptions().setGp(gp); //Sätter i GP i den Genrealla Options som används när nya GP skapas
            gp.getOptions().setGp(gp); //Sätter GP i det aktuella GPt option (olika eftersom varje GP måste komm ihåg sina inställningar (fold).
            simplyfier = new Simplyfier(gp, world, simplificationListeners);
            simplyfier.start();
//            simplyfier.join();
            //simplyfier.start();
            bestSimplyfiedGP = simplyfier.getSimplyfiedGP();
            world.getEnvironment().getOptions().setRUN_CREATE_ALTERNATIVE(false);
        } catch (Exception e) {
            ErrorManager.getInstance().reportError(e);
        }
    }

    public void startSimplification(GP gp) {
        try {
            world.getEnvironment().getOptions().setRUN_SIMPLIFICATION(true);
         //   world.getEnvironment().getOptions().setSIMPLIFICATION_FUNCTION("SimplificationFunction");
            simplyfier = new Simplyfier(gp, world, simplificationListeners);
            simplyfier.start();
//            simplyfier.join();
            //simplyfier.start();
            bestSimplyfiedGP = simplyfier.getSimplyfiedGP();
            world.getEnvironment().getOptions().setRUN_SIMPLIFICATION(false);
        } catch (Exception e) {
            ErrorManager.getInstance().reportError(e);
        }
    }

    public GP getBestGpFromBatch() {
        return getBestGpFromBatch(batchGPs);
    }

    public GP getBestGpFromBatch(ArrayList<GP> gps) {
        GP best;
        best = null;
        for (GP current : gps) {
            if (best == null || best.getFitness() > current.getFitness()) {
                best = current;
            }
        }
        if (best != null) {
            best = (GP) best.clone();
            best.clean();
        }
        return best;
    }

    public double getAverageLength() {
        return world.getAvgPopulationLength();
    }

    public double getAverageFitness() {
        return world.getAvgPopulationFitness();
    }

    public GP getBestGP() {
        return bestGP;
    }

    public int getCurrentGeneration() {
        return currentGeneration;
    }

    public int getCurrentBatch() {
        return currentBatch + 1;
    }

    public int getPersistence() {
        return persistence;
    }

    private synchronized void fireGenerationTerminated() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.generationTerminated(this);
            }
        }
    }

    private synchronized void fireEvolutionTerminated() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.evolutionTerminated(this);
            }
        }
    }

    private synchronized void fireBatchTerminated() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.batchTerminated(this);
            }
        }
    }

    private synchronized void fireEvolutionStarted() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.evolutionStarted(this);
            }
        }
    }

    private synchronized void fireEvolutionProcessTerminated() {
        for (int i = 0; i < evolutionListeners.size(); i++) {
            EvolutionListener l = evolutionListeners.get(i);
            if (l != null) {
                l.evolutionProcessTerminated(this);
            }
        }
        t = null;
        //  this.world=null;
        // this.data.destroyDistanceMatrixAndLookup();
    }

    public String getSummary() {
        return summary;
    }

    public String getSummaryForSelectedGPs() {
        return selectedSummary;
    }

    /**
     * @return the options
     */
    public Options getOptions() {
        return options;
    }

    /**
     * @return the data
     */
    public ArffTableModel getData() {
        return data;
    }

    /**
     * @return the environment
     */
    public Environment getEnvironment() {
        return environment;
    }

    public World getWorld() {
        return world;
    }
    
        public Thread getThread() {
        return t;
    }

    public synchronized void addEvolutionListener(EvolutionListener l) {
        evolutionListeners.add(l);
    }

    public synchronized void removeEvolutionLister(EvolutionListener l) {
        evolutionListeners.remove(l);
    }

    public synchronized void addSimplificationListener(SimplificationListener l) {
        simplificationListeners.add(l);
    }

    public synchronized void removeallEvoltuionListeners() {
        evolutionListeners.clear();
    }

    public synchronized void removeSimplificationListener(SimplificationListener l) {
        simplificationListeners.remove(l);
    }
}
